{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../')\n",
    "from uniperceiver.tokenization import ClipTokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. ImageNet 1k class names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize the class names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We follow CLIP to use the ImageNet-1k class names sourced from Anish Athalye's imagenet-simple-labels.\n",
    "\n",
    "class_names = [\"tench\", \"goldfish\", \"great white shark\", \"tiger shark\", \"hammerhead shark\", \"electric ray\", \"stingray\", \"rooster\", \"hen\", \"ostrich\", \"brambling\", \"goldfinch\", \"house finch\", \"junco\", \"indigo bunting\", \"American robin\", \"bulbul\", \"jay\", \"magpie\", \"chickadee\", \"American dipper\", \"kite (bird of prey)\", \"bald eagle\", \"vulture\", \"great grey owl\", \"fire salamander\", \"smooth newt\", \"newt\", \"spotted salamander\", \"axolotl\", \"American bullfrog\", \"tree frog\", \"tailed frog\", \"loggerhead sea turtle\", \"leatherback sea turtle\", \"mud turtle\", \"terrapin\", \"box turtle\", \"banded gecko\", \"green iguana\", \"Carolina anole\", \"desert grassland whiptail lizard\", \"agama\", \"frilled-necked lizard\", \"alligator lizard\", \"Gila monster\", \"European green lizard\", \"chameleon\", \"Komodo dragon\", \"Nile crocodile\", \"American alligator\", \"triceratops\", \"worm snake\", \"ring-necked snake\", \"eastern hog-nosed snake\", \"smooth green snake\", \"kingsnake\", \"garter snake\", \"water snake\", \"vine snake\", \"night snake\", \"boa constrictor\", \"African rock python\", \"Indian cobra\", \"green mamba\", \"sea snake\", \"Saharan horned viper\", \"eastern diamondback rattlesnake\", \"sidewinder rattlesnake\", \"trilobite\", \"harvestman\", \"scorpion\", \"yellow garden spider\", \"barn spider\", \"European garden spider\", \"southern black widow\", \"tarantula\", \"wolf spider\", \"tick\", \"centipede\", \"black grouse\", \"ptarmigan\", \"ruffed grouse\", \"prairie grouse\", \"peafowl\", \"quail\", \"partridge\", \"african grey parrot\", \"macaw\", \"sulphur-crested cockatoo\", \"lorikeet\", \"coucal\", \"bee eater\", \"hornbill\", \"hummingbird\", \"jacamar\", \"toucan\", \"duck\", \"red-breasted merganser\", \"goose\", \"black swan\", \"tusker\", \"echidna\", \"platypus\", \"wallaby\", \"koala\", \"wombat\", \"jellyfish\", \"sea anemone\", \"brain coral\", \"flatworm\", \"nematode\", \"conch\", \"snail\", \"slug\", \"sea slug\", \"chiton\", \"chambered nautilus\", \"Dungeness crab\", \"rock crab\", \"fiddler crab\", \"red king crab\", \"American lobster\", \"spiny lobster\", \"crayfish\", \"hermit crab\", \"isopod\", \"white stork\", \"black stork\", \"spoonbill\", \"flamingo\", \"little blue heron\", \"great egret\", \"bittern bird\", \"crane bird\", \"limpkin\", \"common gallinule\", \"American coot\", \"bustard\", \"ruddy turnstone\", \"dunlin\", \"common redshank\", \"dowitcher\", \"oystercatcher\", \"pelican\", \"king penguin\", \"albatross\", \"grey whale\", \"killer whale\", \"dugong\", \"sea lion\", \"Chihuahua\", \"Japanese Chin\", \"Maltese\", \"Pekingese\", \"Shih Tzu\", \"King Charles Spaniel\", \"Papillon\", \"toy terrier\", \"Rhodesian Ridgeback\", \"Afghan Hound\", \"Basset Hound\", \"Beagle\", \"Bloodhound\", \"Bluetick Coonhound\", \"Black and Tan Coonhound\", \"Treeing Walker Coonhound\", \"English foxhound\", \"Redbone Coonhound\", \"borzoi\", \"Irish Wolfhound\", \"Italian Greyhound\", \"Whippet\", \"Ibizan Hound\", \"Norwegian Elkhound\", \"Otterhound\", \"Saluki\", \"Scottish Deerhound\", \"Weimaraner\", \"Staffordshire Bull Terrier\", \"American Staffordshire Terrier\", \"Bedlington Terrier\", \"Border Terrier\", \"Kerry Blue Terrier\", \"Irish Terrier\", \"Norfolk Terrier\", \"Norwich Terrier\", \"Yorkshire Terrier\", \"Wire Fox Terrier\", \"Lakeland Terrier\", \"Sealyham Terrier\", \"Airedale Terrier\", \"Cairn Terrier\", \"Australian Terrier\", \"Dandie Dinmont Terrier\", \"Boston Terrier\", \"Miniature Schnauzer\", \"Giant Schnauzer\", \"Standard Schnauzer\", \"Scottish Terrier\", \"Tibetan Terrier\", \"Australian Silky Terrier\", \"Soft-coated Wheaten Terrier\", \"West Highland White Terrier\", \"Lhasa Apso\", \"Flat-Coated Retriever\", \"Curly-coated Retriever\", \"Golden Retriever\", \"Labrador Retriever\", \"Chesapeake Bay Retriever\", \"German Shorthaired Pointer\", \"Vizsla\", \"English Setter\", \"Irish Setter\", \"Gordon Setter\", \"Brittany dog\", \"Clumber Spaniel\", \"English Springer Spaniel\", \"Welsh Springer Spaniel\", \"Cocker Spaniel\", \"Sussex Spaniel\", \"Irish Water Spaniel\", \"Kuvasz\", \"Schipperke\", \"Groenendael dog\", \"Malinois\", \"Briard\", \"Australian Kelpie\", \"Komondor\", \"Old English Sheepdog\", \"Shetland Sheepdog\", \"collie\", \"Border Collie\", \"Bouvier des Flandres dog\", \"Rottweiler\", \"German Shepherd Dog\", \"Dobermann\", \"Miniature Pinscher\", \"Greater Swiss Mountain Dog\", \"Bernese Mountain Dog\", \"Appenzeller Sennenhund\", \"Entlebucher Sennenhund\", \"Boxer\", \"Bullmastiff\", \"Tibetan Mastiff\", \"French Bulldog\", \"Great Dane\", \"St. Bernard\", \"husky\", \"Alaskan Malamute\", \"Siberian Husky\", \"Dalmatian\", \"Affenpinscher\", \"Basenji\", \"pug\", \"Leonberger\", \"Newfoundland dog\", \"Great Pyrenees dog\", \"Samoyed\", \"Pomeranian\", \"Chow Chow\", \"Keeshond\", \"brussels griffon\", \"Pembroke Welsh Corgi\", \"Cardigan Welsh Corgi\", \"Toy Poodle\", \"Miniature Poodle\", \"Standard Poodle\", \"Mexican hairless dog (xoloitzcuintli)\", \"grey wolf\", \"Alaskan tundra wolf\", \"red wolf or maned wolf\", \"coyote\", \"dingo\", \"dhole\", \"African wild dog\", \"hyena\", \"red fox\", \"kit fox\", \"Arctic fox\", \"grey fox\", \"tabby cat\", \"tiger cat\", \"Persian cat\", \"Siamese cat\", \"Egyptian Mau\", \"cougar\", \"lynx\", \"leopard\", \"snow leopard\", \"jaguar\", \"lion\", \"tiger\", \"cheetah\", \"brown bear\", \"American black bear\", \"polar bear\", \"sloth bear\", \"mongoose\", \"meerkat\", \"tiger beetle\", \"ladybug\", \"ground beetle\", \"longhorn beetle\", \"leaf beetle\", \"dung beetle\", \"rhinoceros beetle\", \"weevil\", \"fly\", \"bee\", \"ant\", \"grasshopper\", \"cricket insect\", \"stick insect\", \"cockroach\", \"praying mantis\", \"cicada\", \"leafhopper\", \"lacewing\", \"dragonfly\", \"damselfly\", \"red admiral butterfly\", \"ringlet butterfly\", \"monarch butterfly\", \"small white butterfly\", \"sulphur butterfly\", \"gossamer-winged butterfly\", \"starfish\", \"sea urchin\", \"sea cucumber\", \"cottontail rabbit\", \"hare\", \"Angora rabbit\", \"hamster\", \"porcupine\", \"fox squirrel\", \"marmot\", \"beaver\", \"guinea pig\", \"common sorrel horse\", \"zebra\", \"pig\", \"wild boar\", \"warthog\", \"hippopotamus\", \"ox\", \"water buffalo\", \"bison\", \"ram (adult male sheep)\", \"bighorn sheep\", \"Alpine ibex\", \"hartebeest\", \"impala (antelope)\", \"gazelle\", \"arabian camel\", \"llama\", \"weasel\", \"mink\", \"European polecat\", \"black-footed ferret\", \"otter\", \"skunk\", \"badger\", \"armadillo\", \"three-toed sloth\", \"orangutan\", \"gorilla\", \"chimpanzee\", \"gibbon\", \"siamang\", \"guenon\", \"patas monkey\", \"baboon\", \"macaque\", \"langur\", \"black-and-white colobus\", \"proboscis monkey\", \"marmoset\", \"white-headed capuchin\", \"howler monkey\", \"titi monkey\", \"Geoffroy's spider monkey\", \"common squirrel monkey\", \"ring-tailed lemur\", \"indri\", \"Asian elephant\", \"African bush elephant\", \"red panda\", \"giant panda\", \"snoek fish\", \"eel\", \"silver salmon\", \"rock beauty fish\", \"clownfish\", \"sturgeon\", \"gar fish\", \"lionfish\", \"pufferfish\", \"abacus\", \"abaya\", \"academic gown\", \"accordion\", \"acoustic guitar\", \"aircraft carrier\", \"airliner\", \"airship\", \"altar\", \"ambulance\", \"amphibious vehicle\", \"analog clock\", \"apiary\", \"apron\", \"trash can\", \"assault rifle\", \"backpack\", \"bakery\", \"balance beam\", \"balloon\", \"ballpoint pen\", \"Band-Aid\", \"banjo\", \"baluster / handrail\", \"barbell\", \"barber chair\", \"barbershop\", \"barn\", \"barometer\", \"barrel\", \"wheelbarrow\", \"baseball\", \"basketball\", \"bassinet\", \"bassoon\", \"swimming cap\", \"bath towel\", \"bathtub\", \"station wagon\", \"lighthouse\", \"beaker\", \"military hat (bearskin or shako)\", \"beer bottle\", \"beer glass\", \"bell tower\", \"baby bib\", \"tandem bicycle\", \"bikini\", \"ring binder\", \"binoculars\", \"birdhouse\", \"boathouse\", \"bobsleigh\", \"bolo tie\", \"poke bonnet\", \"bookcase\", \"bookstore\", \"bottle cap\", \"hunting bow\", \"bow tie\", \"brass memorial plaque\", \"bra\", \"breakwater\", \"breastplate\", \"broom\", \"bucket\", \"buckle\", \"bulletproof vest\", \"high-speed train\", \"butcher shop\", \"taxicab\", \"cauldron\", \"candle\", \"cannon\", \"canoe\", \"can opener\", \"cardigan\", \"car mirror\", \"carousel\", \"tool kit\", \"cardboard box / carton\", \"car wheel\", \"automated teller machine\", \"cassette\", \"cassette player\", \"castle\", \"catamaran\", \"CD player\", \"cello\", \"mobile phone\", \"chain\", \"chain-link fence\", \"chain mail\", \"chainsaw\", \"storage chest\", \"chiffonier\", \"bell or wind chime\", \"china cabinet\", \"Christmas stocking\", \"church\", \"movie theater\", \"cleaver\", \"cliff dwelling\", \"cloak\", \"clogs\", \"cocktail shaker\", \"coffee mug\", \"coffeemaker\", \"spiral or coil\", \"combination lock\", \"computer keyboard\", \"candy store\", \"container ship\", \"convertible\", \"corkscrew\", \"cornet\", \"cowboy boot\", \"cowboy hat\", \"cradle\", \"construction crane\", \"crash helmet\", \"crate\", \"infant bed\", \"Crock Pot\", \"croquet ball\", \"crutch\", \"cuirass\", \"dam\", \"desk\", \"desktop computer\", \"rotary dial telephone\", \"diaper\", \"digital clock\", \"digital watch\", \"dining table\", \"dishcloth\", \"dishwasher\", \"disc brake\", \"dock\", \"dog sled\", \"dome\", \"doormat\", \"drilling rig\", \"drum\", \"drumstick\", \"dumbbell\", \"Dutch oven\", \"electric fan\", \"electric guitar\", \"electric locomotive\", \"entertainment center\", \"envelope\", \"espresso machine\", \"face powder\", \"feather boa\", \"filing cabinet\", \"fireboat\", \"fire truck\", \"fire screen\", \"flagpole\", \"flute\", \"folding chair\", \"football helmet\", \"forklift\", \"fountain\", \"fountain pen\", \"four-poster bed\", \"freight car\", \"French horn\", \"frying pan\", \"fur coat\", \"garbage truck\", \"gas mask or respirator\", \"gas pump\", \"goblet\", \"go-kart\", \"golf ball\", \"golf cart\", \"gondola\", \"gong\", \"gown\", \"grand piano\", \"greenhouse\", \"radiator grille\", \"grocery store\", \"guillotine\", \"hair clip\", \"hair spray\", \"half-track\", \"hammer\", \"hamper\", \"hair dryer\", \"hand-held computer\", \"handkerchief\", \"hard disk drive\", \"harmonica\", \"harp\", \"combine harvester\", \"hatchet\", \"holster\", \"home theater\", \"honeycomb\", \"hook\", \"hoop skirt\", \"gymnastic horizontal bar\", \"horse-drawn vehicle\", \"hourglass\", \"iPod\", \"clothes iron\", \"carved pumpkin\", \"jeans\", \"jeep\", \"T-shirt\", \"jigsaw puzzle\", \"rickshaw\", \"joystick\", \"kimono\", \"knee pad\", \"knot\", \"lab coat\", \"ladle\", \"lampshade\", \"laptop computer\", \"lawn mower\", \"lens cap\", \"letter opener\", \"library\", \"lifeboat\", \"lighter\", \"limousine\", \"ocean liner\", \"lipstick\", \"slip-on shoe\", \"lotion\", \"music speaker\", \"loupe magnifying glass\", \"sawmill\", \"magnetic compass\", \"messenger bag\", \"mailbox\", \"tights\", \"one-piece bathing suit\", \"manhole cover\", \"maraca\", \"marimba\", \"mask\", \"matchstick\", \"maypole\", \"maze\", \"measuring cup\", \"medicine cabinet\", \"megalith\", \"microphone\", \"microwave oven\", \"military uniform\", \"milk can\", \"minibus\", \"miniskirt\", \"minivan\", \"missile\", \"mitten\", \"mixing bowl\", \"mobile home\", \"ford model t\", \"modem\", \"monastery\", \"monitor\", \"moped\", \"mortar and pestle\", \"graduation cap\", \"mosque\", \"mosquito net\", \"vespa\", \"mountain bike\", \"tent\", \"computer mouse\", \"mousetrap\", \"moving van\", \"muzzle\", \"metal nail\", \"neck brace\", \"necklace\", \"baby pacifier\", \"notebook computer\", \"obelisk\", \"oboe\", \"ocarina\", \"odometer\", \"oil filter\", \"pipe organ\", \"oscilloscope\", \"overskirt\", \"bullock cart\", \"oxygen mask\", \"product packet / packaging\", \"paddle\", \"paddle wheel\", \"padlock\", \"paintbrush\", \"pajamas\", \"palace\", \"pan flute\", \"paper towel\", \"parachute\", \"parallel bars\", \"park bench\", \"parking meter\", \"railroad car\", \"patio\", \"payphone\", \"pedestal\", \"pencil case\", \"pencil sharpener\", \"perfume\", \"Petri dish\", \"photocopier\", \"plectrum\", \"Pickelhaube\", \"picket fence\", \"pickup truck\", \"pier\", \"piggy bank\", \"pill bottle\", \"pillow\", \"ping-pong ball\", \"pinwheel\", \"pirate ship\", \"drink pitcher\", \"block plane\", \"planetarium\", \"plastic bag\", \"plate rack\", \"farm plow\", \"plunger\", \"Polaroid camera\", \"pole\", \"police van\", \"poncho\", \"pool table\", \"soda bottle\", \"plant pot\", \"potter's wheel\", \"power drill\", \"prayer rug\", \"printer\", \"prison\", \"missile\", \"projector\", \"hockey puck\", \"punching bag\", \"purse\", \"quill\", \"quilt\", \"race car\", \"racket\", \"radiator\", \"radio\", \"radio telescope\", \"rain barrel\", \"recreational vehicle\", \"fishing casting reel\", \"reflex camera\", \"refrigerator\", \"remote control\", \"restaurant\", \"revolver\", \"rifle\", \"rocking chair\", \"rotisserie\", \"eraser\", \"rugby ball\", \"ruler measuring stick\", \"sneaker\", \"safe\", \"safety pin\", \"salt shaker\", \"sandal\", \"sarong\", \"saxophone\", \"scabbard\", \"weighing scale\", \"school bus\", \"schooner\", \"scoreboard\", \"CRT monitor\", \"screw\", \"screwdriver\", \"seat belt\", \"sewing machine\", \"shield\", \"shoe store\", \"shoji screen / room divider\", \"shopping basket\", \"shopping cart\", \"shovel\", \"shower cap\", \"shower curtain\", \"ski\", \"balaclava ski mask\", \"sleeping bag\", \"slide rule\", \"sliding door\", \"slot machine\", \"snorkel\", \"snowmobile\", \"snowplow\", \"soap dispenser\", \"soccer ball\", \"sock\", \"solar thermal collector\", \"sombrero\", \"soup bowl\", \"keyboard space bar\", \"space heater\", \"space shuttle\", \"spatula\", \"motorboat\", \"spider web\", \"spindle\", \"sports car\", \"spotlight\", \"stage\", \"steam locomotive\", \"through arch bridge\", \"steel drum\", \"stethoscope\", \"scarf\", \"stone wall\", \"stopwatch\", \"stove\", \"strainer\", \"tram\", \"stretcher\", \"couch\", \"stupa\", \"submarine\", \"suit\", \"sundial\", \"sunglasses\", \"sunglasses\", \"sunscreen\", \"suspension bridge\", \"mop\", \"sweatshirt\", \"swim trunks / shorts\", \"swing\", \"electrical switch\", \"syringe\", \"table lamp\", \"tank\", \"tape player\", \"teapot\", \"teddy bear\", \"television\", \"tennis ball\", \"thatched roof\", \"front curtain\", \"thimble\", \"threshing machine\", \"throne\", \"tile roof\", \"toaster\", \"tobacco shop\", \"toilet seat\", \"torch\", \"totem pole\", \"tow truck\", \"toy store\", \"tractor\", \"semi-trailer truck\", \"tray\", \"trench coat\", \"tricycle\", \"trimaran\", \"tripod\", \"triumphal arch\", \"trolleybus\", \"trombone\", \"hot tub\", \"turnstile\", \"typewriter keyboard\", \"umbrella\", \"unicycle\", \"upright piano\", \"vacuum cleaner\", \"vase\", \"vaulted or arched ceiling\", \"velvet fabric\", \"vending machine\", \"vestment\", \"viaduct\", \"violin\", \"volleyball\", \"waffle iron\", \"wall clock\", \"wallet\", \"wardrobe\", \"military aircraft\", \"sink\", \"washing machine\", \"water bottle\", \"water jug\", \"water tower\", \"whiskey jug\", \"whistle\", \"hair wig\", \"window screen\", \"window shade\", \"Windsor tie\", \"wine bottle\", \"airplane wing\", \"wok\", \"wooden spoon\", \"wool\", \"split-rail fence\", \"shipwreck\", \"sailboat\", \"yurt\", \"website\", \"comic book\", \"crossword\", \"traffic or street sign\", \"traffic light\", \"dust jacket\", \"menu\", \"plate\", \"guacamole\", \"consomme\", \"hot pot\", \"trifle\", \"ice cream\", \"popsicle\", \"baguette\", \"bagel\", \"pretzel\", \"cheeseburger\", \"hot dog\", \"mashed potatoes\", \"cabbage\", \"broccoli\", \"cauliflower\", \"zucchini\", \"spaghetti squash\", \"acorn squash\", \"butternut squash\", \"cucumber\", \"artichoke\", \"bell pepper\", \"cardoon\", \"mushroom\", \"Granny Smith apple\", \"strawberry\", \"orange\", \"lemon\", \"fig\", \"pineapple\", \"banana\", \"jackfruit\", \"cherimoya (custard apple)\", \"pomegranate\", \"hay\", \"carbonara\", \"chocolate syrup\", \"dough\", \"meatloaf\", \"pizza\", \"pot pie\", \"burrito\", \"red wine\", \"espresso\", \"tea cup\", \"eggnog\", \"mountain\", \"bubble\", \"cliff\", \"coral reef\", \"geyser\", \"lakeshore\", \"promontory\", \"sandbar\", \"beach\", \"valley\", \"volcano\", \"baseball player\", \"bridegroom\", \"scuba diver\", \"rapeseed\", \"daisy\", \"yellow lady's slipper\", \"corn\", \"acorn\", \"rose hip\", \"horse chestnut seed\", \"coral fungus\", \"agaric\", \"gyromitra\", \"stinkhorn mushroom\", \"earth star fungus\", \"hen of the woods mushroom\", \"bolete\", \"corn cob\", \"toilet paper\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = ClipTokenizer()\n",
    "class_names_tokenized = [torch.tensor(tokenizer.encode(x + \" <|endoftext|>\")) for x in class_names]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "shared_target_set = {\n",
    "    'data': class_names_tokenized,\n",
    "    'modality': 'text'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the target set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This file location should be specified in the \"SHARED_TARGETS\" section of the config file.\n",
    "with open('/nfs/zhujinguo/datasets/open_source_dataset/imagenet_class_name_CLIP_with_endoftext.pkl', 'wb') as f:\n",
    "    pickle.dump(shared_target_set, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. ImageNet 22k class names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For ImageNet 22k, we use the ImageNet 1k class names as the first 1000 categories, and use the WordNet synonyms for the remaining categories.\n",
    "with open('/nfs/zhujinguo/datasets/open_source_dataset/imagenet_22k_class_names.pkl', 'rb') as f:\n",
    "    class_name_mapping = pickle.load(f)\n",
    "\n",
    "class_names = [c[1] for c in class_name_mapping]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = ClipTokenizer()\n",
    "class_names_tokenized = [[torch.tensor(tokenizer.encode(x + \" <|endoftext|>\")) for x in c] for c in class_names]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "shared_target_set = {\n",
    "    'data': class_names_tokenized,\n",
    "    'modality': 'text'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the target set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This file location should be specified in the \"SHARED_TARGETS\" section of the config file.\n",
    "with open('/nfs/zhujinguo/datasets/open_source_dataset/imagenet_22k_class_name_CLIP_with_endoftext.pkl', 'wb') as f:\n",
    "    pickle.dump(shared_target_set, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Vocabulary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize the vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = ClipTokenizer()\n",
    "eot = tokenizer.encoder['<|endoftext|>']\n",
    "vocabulary_tokenized = [torch.tensor([v, eot]) for v in tokenizer.encoder.values]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "shared_target_set = {\n",
    "    'data': vocabulary_tokenized,\n",
    "    'modality': 'text'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the target set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/nfs/zhujinguo/datasets/open_source_dataset/vocabulary_CLIP_with_endoftext.pkl', 'wb') as f:\n",
    "    pickle.dump(shared_target_set, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Kinetics-400 class names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize the class names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Class names obtained from https://gist.github.com/willprice/f19da185c9c5f32847134b87c1960769\n",
    "\n",
    "class_names = ['abseiling', 'air drumming', 'answering questions', 'applauding', 'applying cream', 'archery', 'arm wrestling', 'arranging flowers', 'assembling computer', 'auctioning', 'baby waking up', 'baking cookies', 'balloon blowing', 'bandaging', 'barbequing', 'bartending', 'beatboxing', 'bee keeping', 'belly dancing', 'bench pressing', 'bending back', 'bending metal', 'biking through snow', 'blasting sand', 'blowing glass', 'blowing leaves', 'blowing nose', 'blowing out candles', 'bobsledding', 'bookbinding', 'bouncing on trampoline', 'bowling', 'braiding hair', 'breading or breadcrumbing', 'breakdancing', 'brush painting', 'brushing hair', 'brushing teeth', 'building cabinet', 'building shed', 'bungee jumping', 'busking', 'canoeing or kayaking', 'capoeira', 'carrying baby', 'cartwheeling', 'carving pumpkin', 'catching fish', 'catching or throwing baseball', 'catching or throwing frisbee', 'catching or throwing softball', 'celebrating', 'changing oil', 'changing wheel', 'checking tires', 'cheerleading', 'chopping wood', 'clapping', 'clay pottery making', 'clean and jerk', 'cleaning floor', 'cleaning gutters', 'cleaning pool', 'cleaning shoes', 'cleaning toilet', 'cleaning windows', 'climbing a rope', 'climbing ladder', 'climbing tree', 'contact juggling', 'cooking chicken', 'cooking egg', 'cooking on campfire', 'cooking sausages', 'counting money', 'country line dancing', 'cracking neck', 'crawling baby', 'crossing river', 'crying', 'curling hair', 'cutting nails', 'cutting pineapple', 'cutting watermelon', 'dancing ballet', 'dancing charleston', 'dancing gangnam style', 'dancing macarena', 'deadlifting', 'decorating the christmas tree', 'digging', 'dining', 'disc golfing', 'diving cliff', 'dodgeball', 'doing aerobics', 'doing laundry', 'doing nails', 'drawing', 'dribbling basketball', 'drinking', 'drinking beer', 'drinking shots', 'driving car', 'driving tractor', 'drop kicking', 'drumming fingers', 'dunking basketball', 'dying hair', 'eating burger', 'eating cake', 'eating carrots', 'eating chips', 'eating doughnuts', 'eating hotdog', 'eating ice cream', 'eating spaghetti', 'eating watermelon', 'egg hunting', 'exercising arm', 'exercising with an exercise ball', 'extinguishing fire', 'faceplanting', 'feeding birds', 'feeding fish', 'feeding goats', 'filling eyebrows', 'finger snapping', 'fixing hair', 'flipping pancake', 'flying kite', 'folding clothes', 'folding napkins', 'folding paper', 'front raises', 'frying vegetables', 'garbage collecting', 'gargling', 'getting a haircut', 'getting a tattoo', 'giving or receiving award', 'golf chipping', 'golf driving', 'golf putting', 'grinding meat', 'grooming dog', 'grooming horse', 'gymnastics tumbling', 'hammer throw', 'headbanging', 'headbutting', 'high jump', 'high kick', 'hitting baseball', 'hockey stop', 'holding snake', 'hopscotch', 'hoverboarding', 'hugging', 'hula hooping', 'hurdling', 'hurling (sport)', 'ice climbing', 'ice fishing', 'ice skating', 'ironing', 'javelin throw', 'jetskiing', 'jogging', 'juggling balls', 'juggling fire', 'juggling soccer ball', 'jumping into pool', 'jumpstyle dancing', 'kicking field goal', 'kicking soccer ball', 'kissing', 'kitesurfing', 'knitting', 'krumping', 'laughing', 'laying bricks', 'long jump', 'lunge', 'making a cake', 'making a sandwich', 'making bed', 'making jewelry', 'making pizza', 'making snowman', 'making sushi', 'making tea', 'marching', 'massaging back', 'massaging feet', 'massaging legs', \"massaging person's head\", 'milking cow', 'mopping floor', 'motorcycling', 'moving furniture', 'mowing lawn', 'news anchoring', 'opening bottle', 'opening present', 'paragliding', 'parasailing', 'parkour', 'passing American football (in game)', 'passing American football (not in game)', 'peeling apples', 'peeling potatoes', 'petting animal (not cat)', 'petting cat', 'picking fruit', 'planting trees', 'plastering', 'playing accordion', 'playing badminton', 'playing bagpipes', 'playing basketball', 'playing bass guitar', 'playing cards', 'playing cello', 'playing chess', 'playing clarinet', 'playing controller', 'playing cricket', 'playing cymbals', 'playing didgeridoo', 'playing drums', 'playing flute', 'playing guitar', 'playing harmonica', 'playing harp', 'playing ice hockey', 'playing keyboard', 'playing kickball', 'playing monopoly', 'playing organ', 'playing paintball', 'playing piano', 'playing poker', 'playing recorder', 'playing saxophone', 'playing squash or racquetball', 'playing tennis', 'playing trombone', 'playing trumpet', 'playing ukulele', 'playing violin', 'playing volleyball', 'playing xylophone', 'pole vault', 'presenting weather forecast', 'pull ups', 'pumping fist', 'pumping gas', 'punching bag', 'punching person (boxing)', 'push up', 'pushing car', 'pushing cart', 'pushing wheelchair', 'reading book', 'reading newspaper', 'recording music', 'riding a bike', 'riding camel', 'riding elephant', 'riding mechanical bull', 'riding mountain bike', 'riding mule', 'riding or walking with horse', 'riding scooter', 'riding unicycle', 'ripping paper', 'robot dancing', 'rock climbing', 'rock scissors paper', 'roller skating', 'running on treadmill', 'sailing', 'salsa dancing', 'sanding floor', 'scrambling eggs', 'scuba diving', 'setting table', 'shaking hands', 'shaking head', 'sharpening knives', 'sharpening pencil', 'shaving head', 'shaving legs', 'shearing sheep', 'shining shoes', 'shooting basketball', 'shooting goal (soccer)', 'shot put', 'shoveling snow', 'shredding paper', 'shuffling cards', 'side kick', 'sign language interpreting', 'singing', 'situp', 'skateboarding', 'ski jumping', 'skiing (not slalom or crosscountry)', 'skiing crosscountry', 'skiing slalom', 'skipping rope', 'skydiving', 'slacklining', 'slapping', 'sled dog racing', 'smoking', 'smoking hookah', 'snatch weight lifting', 'sneezing', 'sniffing', 'snorkeling', 'snowboarding', 'snowkiting', 'snowmobiling', 'somersaulting', 'spinning poi', 'spray painting', 'spraying', 'springboard diving', 'squat', 'sticking tongue out', 'stomping grapes', 'stretching arm', 'stretching leg', 'strumming guitar', 'surfing crowd', 'surfing water', 'sweeping floor', 'swimming backstroke', 'swimming breast stroke', 'swimming butterfly stroke', 'swing dancing', 'swinging legs', 'swinging on something', 'sword fighting', 'tai chi', 'taking a shower', 'tango dancing', 'tap dancing', 'tapping guitar', 'tapping pen', 'tasting beer', 'tasting food', 'testifying', 'texting', 'throwing axe', 'throwing ball', 'throwing discus', 'tickling', 'tobogganing', 'tossing coin', 'tossing salad', 'training dog', 'trapezing', 'trimming or shaving beard', 'trimming trees', 'triple jump', 'tying bow tie', 'tying knot (not on a tie)', 'tying tie', 'unboxing', 'unloading truck', 'using computer', 'using remote controller (not gaming)', 'using segway', 'vault', 'waiting in line', 'walking the dog', 'washing dishes', 'washing feet', 'washing hair', 'washing hands', 'water skiing', 'water sliding', 'watering plants', 'waxing back', 'waxing chest', 'waxing eyebrows', 'waxing legs', 'weaving basket', 'welding', 'whistling', 'windsurfing', 'wrapping present', 'wrestling', 'writing', 'yawning', 'yoga', 'zumba']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = ClipTokenizer()\n",
    "class_names_tokenized = [torch.tensor(tokenizer.encode(x + \" <|endoftext|>\")) for x in class_names]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "shared_target_set = {\n",
    "    'data': class_names_tokenized,\n",
    "    'modality': 'text'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the target set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This file location should be specified in the \"SHARED_TARGETS\" section of the config file.\n",
    "with open('/nfs/zhujinguo/datasets/open_source_dataset/k400_class_name_CLIP_with_endoftext.pkl', 'wb') as f:\n",
    "    pickle.dump(shared_target_set, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Kinetics-700 class names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize the class names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Class names obtained from https://gist.github.com/willprice/f19da185c9c5f32847134b87c1960769\n",
    "\n",
    "class_names = ['abseiling', 'acting in play', 'adjusting glasses', 'air drumming', 'alligator wrestling', 'answering questions', 'applauding', 'applying cream', 'archaeological excavation', 'archery', 'arguing', 'arm wrestling', 'arranging flowers', 'arresting', 'assembling bicycle', 'assembling computer', 'attending conference', 'auctioning', 'baby waking up', 'backflip (human)', 'baking cookies', 'bandaging', 'barbequing', 'bartending', 'base jumping', 'bathing dog', 'battle rope training', 'beatboxing', 'bee keeping', 'being excited', 'being in zero gravity', 'belly dancing', 'bench pressing', 'bending back', 'bending metal', 'biking through snow', 'blasting sand', 'blending fruit', 'blowdrying hair', 'blowing bubble gum', 'blowing glass', 'blowing leaves', 'blowing nose', 'blowing out candles', 'bobsledding', 'bodysurfing', 'bookbinding', 'bottling', 'bouncing ball (not juggling)', 'bouncing on bouncy castle', 'bouncing on trampoline', 'bowling', 'braiding hair', 'breading or breadcrumbing', 'breakdancing', 'breaking boards', 'breaking glass', 'breathing fire', 'brush painting', 'brushing floor', 'brushing hair', 'brushing teeth', 'building cabinet', 'building lego', 'building sandcastle', 'building shed', 'bulldozing', 'bungee jumping', 'burping', 'busking', 'calculating', 'calligraphy', 'canoeing or kayaking', 'capoeira', 'capsizing', 'card stacking', 'card throwing', 'carrying baby', 'carrying weight', 'cartwheeling', 'carving ice', 'carving marble', 'carving pumpkin', 'carving wood with a knife', 'casting fishing line', 'catching fish', 'catching or throwing baseball', 'catching or throwing frisbee', 'catching or throwing softball', 'celebrating', 'changing gear in car', 'changing oil', 'changing wheel (not on bike)', 'chasing', 'checking tires', 'checking watch', 'cheerleading', 'chewing gum', 'chiseling stone', 'chiseling wood', 'chopping meat', 'chopping wood', 'clam digging', 'clapping', 'clay pottery making', 'clean and jerk', 'cleaning gutters', 'cleaning pool', 'cleaning shoes', 'cleaning toilet', 'cleaning windows', 'climbing a rope', 'climbing ladder', 'climbing tree', 'closing door', 'coloring in', 'combing hair', 'contact juggling', 'contorting', 'cooking chicken', 'cooking egg', 'cooking on campfire', 'cooking sausages (not on barbeque)', 'cooking scallops', 'cosplaying', 'coughing', 'counting money', 'country line dancing', 'cracking back', 'cracking knuckles', 'cracking neck', 'crawling baby', 'crocheting', 'crossing eyes', 'crossing river', 'crying', 'cumbia', 'curling (sport)', 'curling eyelashes', 'curling hair', 'cutting apple', 'cutting cake', 'cutting nails', 'cutting orange', 'cutting pineapple', 'cutting watermelon', 'dancing ballet', 'dancing charleston', 'dancing gangnam style', 'dancing macarena', 'deadlifting', 'dealing cards', 'decorating the christmas tree', 'decoupage', 'delivering mail', 'digging', 'dining', 'directing traffic', 'disc golfing', 'diving cliff', 'docking boat', 'dodgeball', 'doing aerobics', 'doing jigsaw puzzle', 'doing laundry', 'doing nails', 'doing sudoku', 'drawing', 'dribbling basketball', 'drinking shots', 'driving car', 'driving tractor', 'drooling', 'drop kicking', 'drumming fingers', 'dumpster diving', 'dunking basketball', 'dyeing eyebrows', 'dyeing hair', 'eating burger', 'eating cake', 'eating carrots', 'eating chips', 'eating doughnuts', 'eating hotdog', 'eating ice cream', 'eating nachos', 'eating spaghetti', 'eating watermelon', 'egg hunting', 'embroidering', 'entering church', 'exercising arm', 'exercising with an exercise ball', 'extinguishing fire', 'faceplanting', 'falling off bike', 'falling off chair', 'feeding birds', 'feeding fish', 'feeding goats', 'fencing (sport)', 'fidgeting', 'filling cake', 'filling eyebrows', 'finger snapping', 'fixing bicycle', 'fixing hair', 'flint knapping', 'flipping bottle', 'flipping pancake', 'fly tying', 'flying kite', 'folding clothes', 'folding napkins', 'folding paper', 'front raises', 'frying vegetables', 'gargling', 'geocaching', 'getting a haircut', 'getting a piercing', 'getting a tattoo', 'giving or receiving award', 'gold panning', 'golf chipping', 'golf driving', 'golf putting', 'gospel singing in church', 'grinding meat', 'grooming cat', 'grooming dog', 'grooming horse', 'gymnastics tumbling', 'hammer throw', 'hand washing clothes', 'head stand', 'headbanging', 'headbutting', 'helmet diving', 'herding cattle', 'high fiving', 'high jump', 'high kick', 'historical reenactment', 'hitting baseball', 'hockey stop', 'holding snake', 'home roasting coffee', 'hopscotch', 'hoverboarding', 'huddling', 'hugging (not baby)', 'hugging baby', 'hula hooping', 'hurdling', 'hurling (sport)', 'ice climbing', 'ice fishing', 'ice skating', 'ice swimming', 'inflating balloons', 'installing carpet', 'ironing', 'ironing hair', 'javelin throw', 'jaywalking', 'jetskiing', 'jogging', 'juggling balls', 'juggling fire', 'juggling soccer ball', 'jumping bicycle', 'jumping into pool', 'jumping jacks', 'jumping sofa', 'jumpstyle dancing', 'karaoke', 'kicking field goal', 'kicking soccer ball', 'kissing', 'kitesurfing', 'knitting', 'krumping', 'land sailing', 'laughing', 'lawn mower racing', 'laying bricks', 'laying concrete', 'laying decking', 'laying stone', 'laying tiles', 'leatherworking', 'letting go of balloon', 'licking', 'lifting hat', 'lighting candle', 'lighting fire', 'listening with headphones', 'lock picking', 'long jump', 'longboarding', 'looking at phone', 'looking in mirror', 'luge', 'lunge', 'making a cake', 'making a sandwich', 'making balloon shapes', 'making bubbles', 'making cheese', 'making horseshoes', 'making jewelry', 'making latte art', 'making paper aeroplanes', 'making pizza', 'making slime', 'making snowman', 'making sushi', 'making tea', 'making the bed', 'marching', 'marriage proposal', 'massaging back', 'massaging feet', 'massaging legs', 'massaging neck', \"massaging person's head\", 'metal detecting', 'milking cow', 'milking goat', 'mixing colours', 'moon walking', 'mopping floor', 'mosh pit dancing', 'motorcycling', 'mountain climber (exercise)', 'moving baby', 'moving child', 'moving furniture', 'mowing lawn', 'mushroom foraging', 'needle felting', 'news anchoring', 'opening bottle (not wine)', 'opening coconuts', 'opening door', 'opening present', 'opening refrigerator', 'opening wine bottle', 'packing', 'paragliding', 'parasailing', 'parkour', 'passing American football (in game)', 'passing American football (not in game)', 'passing soccer ball', 'peeling apples', 'peeling banana', 'peeling potatoes', 'person collecting garbage', 'petting animal (not cat)', 'petting cat', 'petting horse', 'photobombing', 'photocopying', 'picking apples', 'picking blueberries', 'pillow fight', 'pinching', 'pirouetting', 'planing wood', 'planting trees', 'plastering', 'playing accordion', 'playing american football', 'playing badminton', 'playing bagpipes', 'playing basketball', 'playing bass guitar', 'playing beer pong', 'playing billiards', 'playing blackjack', 'playing cards', 'playing cello', 'playing checkers', 'playing chess', 'playing clarinet', 'playing controller', 'playing cricket', 'playing cymbals', 'playing darts', 'playing didgeridoo', 'playing dominoes', 'playing drums', 'playing field hockey', 'playing flute', 'playing gong', 'playing guitar', 'playing hand clapping games', 'playing harmonica', 'playing harp', 'playing ice hockey', 'playing keyboard', 'playing kickball', 'playing laser tag', 'playing lute', 'playing mahjong', 'playing maracas', 'playing marbles', 'playing monopoly', 'playing netball', 'playing nose flute', 'playing oboe', 'playing ocarina', 'playing organ', 'playing paintball', 'playing pan pipes', 'playing piano', 'playing piccolo', 'playing pinball', 'playing ping pong', 'playing poker', 'playing polo', 'playing recorder', 'playing road hockey', 'playing rounders', 'playing rubiks cube', 'playing saxophone', 'playing scrabble', 'playing shuffleboard', 'playing slot machine', 'playing squash or racquetball', 'playing tennis', 'playing trombone', 'playing trumpet', 'playing ukulele', 'playing violin', 'playing volleyball', 'playing with trains', 'playing xylophone', 'poaching eggs', 'poking bellybutton', 'pole vault', 'polishing furniture', 'polishing metal', 'popping balloons', 'pouring beer', 'pouring milk', 'pouring wine', 'preparing salad', 'presenting weather forecast', 'pretending to be a statue', 'pull ups', 'pulling espresso shot', 'pulling rope (game)', 'pumping fist', 'pumping gas', 'punching bag', 'punching person (boxing)', 'push up', 'pushing car', 'pushing cart', 'pushing wheelbarrow', 'pushing wheelchair', 'putting in contact lenses', 'putting on eyeliner', 'putting on foundation', 'putting on lipstick', 'putting on mascara', 'putting on sari', 'putting on shoes', 'putting wallpaper on wall', 'raising eyebrows', 'reading book', 'reading newspaper', 'recording music', 'repairing puncture', 'riding a bike', 'riding camel', 'riding elephant', 'riding mechanical bull', 'riding mule', 'riding or walking with horse', 'riding scooter', 'riding snow blower', 'riding unicycle', 'ripping paper', 'roasting marshmallows', 'roasting pig', 'robot dancing', 'rock climbing', 'rock scissors paper', 'roller skating', 'rolling eyes', 'rolling pastry', 'rope pushdown', 'running on treadmill', 'sailing', 'salsa dancing', 'saluting', 'sanding floor', 'sanding wood', 'sausage making', 'sawing wood', 'scrambling eggs', 'scrapbooking', 'scrubbing face', 'scuba diving', 'seasoning food', 'separating eggs', 'setting table', 'sewing', 'shaking hands', 'shaking head', 'shaping bread dough', 'sharpening knives', 'sharpening pencil', 'shaving head', 'shaving legs', 'shearing sheep', 'shining flashlight', 'shining shoes', 'shoot dance', 'shooting basketball', 'shooting goal (soccer)', 'shooting off fireworks', 'shopping', 'shot put', 'shouting', 'shoveling snow', 'shredding paper', 'shucking oysters', 'shuffling cards', 'shuffling feet', 'side kick', 'sieving', 'sign language interpreting', 'silent disco', 'singing', 'sipping cup', 'situp', 'skateboarding', 'ski ballet', 'ski jumping', 'skiing crosscountry', 'skiing mono', 'skiing slalom', 'skipping rope', 'skipping stone', 'skydiving', 'slacklining', 'slapping', 'sled dog racing', 'sleeping', 'slicing onion', 'smashing', 'smelling feet', 'smoking', 'smoking hookah', 'smoking pipe', 'snatch weight lifting', 'sneezing', 'snorkeling', 'snowboarding', 'snowkiting', 'snowmobiling', 'somersaulting', 'spelunking', 'spinning plates', 'spinning poi', 'splashing water', 'spray painting', 'spraying', 'springboard diving', 'square dancing', 'squat', 'squeezing orange', 'stacking cups', 'stacking dice', 'standing on hands', 'staring', 'steer roping', 'steering car', 'sticking tongue out', 'stomping grapes', 'stretching arm', 'stretching leg', 'sucking lolly', 'surfing crowd', 'surfing water', 'surveying', 'sweeping floor', 'swimming backstroke', 'swimming breast stroke', 'swimming butterfly stroke', 'swimming front crawl', 'swimming with dolphins', 'swimming with sharks', 'swing dancing', 'swinging baseball bat', 'swinging on something', 'sword fighting', 'sword swallowing', 'tackling', 'tagging graffiti', 'tai chi', 'taking photo', 'talking on cell phone', 'tango dancing', 'tap dancing', 'tapping guitar', 'tapping pen', 'tasting beer', 'tasting food', 'tasting wine', 'testifying', 'texting', 'threading needle', 'throwing axe', 'throwing ball (not baseball or American football)', 'throwing discus', 'throwing knife', 'throwing snowballs', 'throwing tantrum', 'throwing water balloon', 'tickling', 'tie dying', 'tightrope walking', 'tiptoeing', 'tobogganing', 'tossing coin', 'tossing salad', 'training dog', 'trapezing', 'treating wood', 'trimming or shaving beard', 'trimming shrubs', 'trimming trees', 'triple jump', 'twiddling fingers', 'tying bow tie', 'tying knot (not on a tie)', 'tying necktie', 'tying shoe laces', 'unboxing', 'uncorking champagne', 'unloading truck', 'using a microscope', 'using a paint roller', 'using a power drill', 'using a sledge hammer', 'using a wrench', 'using atm', 'using bagging machine', 'using circular saw', 'using inhaler', 'using megaphone', 'using puppets', 'using remote controller (not gaming)', 'using segway', 'vacuuming car', 'vacuuming floor', 'visiting the zoo', 'wading through mud', 'wading through water', 'waiting in line', 'waking up', 'walking on stilts', 'walking the dog', 'walking through snow', 'walking with crutches', 'washing dishes', 'washing feet', 'washing hair', 'washing hands', 'watching tv', 'water skiing', 'water sliding', 'watering plants', 'waving hand', 'waxing armpits', 'waxing back', 'waxing chest', 'waxing eyebrows', 'waxing legs', 'weaving basket', 'weaving fabric', 'welding', 'whistling', 'windsurfing', 'winking', 'wood burning (art)', 'wrapping present', 'wrestling', 'writing', 'yarn spinning', 'yawning', 'yoga', 'zumba']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = ClipTokenizer()\n",
    "class_names_tokenized = [torch.tensor(tokenizer.encode(x + \" <|endoftext|>\")) for x in class_names]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "shared_target_set = {\n",
    "    'data': class_names_tokenized,\n",
    "    'modality': 'text'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the target set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This file location should be specified in the \"SHARED_TARGETS\" section of the config file.\n",
    "with open('/nfs/zhujinguo/datasets/open_source_dataset/k700_class_name_CLIP_with_endoftext.pkl', 'wb') as f:\n",
    "    pickle.dump(shared_target_set, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Moments in time class names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_names = ['adult female singing', 'adult female speaking', 'adult male singing', 'adult male speaking', 'aiming', 'applauding', 'arresting', 'ascending', 'asking', 'assembling', 'attacking', 'autographing', 'baking', 'balancing', 'baptizing', 'barbecuing', 'barking', 'bathing', 'bending', 'bicycling', 'biting', 'blocking', 'blowing', 'boarding', 'boating', 'boiling', 'bouncing', 'bowing', 'bowling', 'boxing', 'breaking', 'brushing', 'bubbling', 'building', 'bulldozing', 'burning', 'burying', 'buttoning', 'buying', 'calling', 'camping', 'carrying', 'carving', 'catching', 'celebrating', 'chasing', 'cheering', 'cheerleading', 'chewing', 'child singing', 'child speaking', 'chopping', 'clapping', 'clawing', 'cleaning', 'clearing', 'climbing', 'clinging', 'clipping', 'closing', 'coaching', 'colliding', 'combing', 'combusting', 'competing', 'constructing', 'cooking', 'coughing', 'covering', 'cracking', 'crafting', 'cramming', 'crashing', 'crawling', 'crouching', 'crushing', 'crying', 'cuddling', 'cutting', 'dancing', 'descending', 'destroying', 'digging', 'dining', 'dipping', 'discussing', 'diving', 'dragging', 'draining', 'drawing', 'drenching', 'dressing', 'drilling', 'drinking', 'dripping', 'driving', 'dropping', 'drumming', 'drying', 'dunking', 'dusting', 'eating', 'emptying', 'entering', 'erupting', 'exercising', 'exiting', 'extinguishing', 'falling', 'feeding', 'fencing', 'fighting', 'filling', 'filming', 'fishing', 'flicking', 'flipping', 'floating', 'flooding', 'flowing', 'flying', 'folding', 'frowning', 'frying', 'fueling', 'gambling', 'gardening', 'giggling', 'giving', 'grilling', 'grinning', 'gripping', 'grooming', 'guarding', 'hammering', 'handcuffing', 'handwriting', 'hanging', 'hiking', 'hitchhiking', 'hitting', 'howling', 'hugging', 'hunting', 'imitating', 'inflating', 'injecting', 'instructing', 'interviewing', 'jogging', 'joining', 'juggling', 'jumping', 'kicking', 'kissing', 'kneeling', 'knitting', 'knocking', 'landing', 'laughing', 'launching', 'leaking', 'leaning', 'leaping', 'lecturing', 'licking', 'lifting', 'loading', 'locking', 'manicuring', 'marching', 'marrying', 'massaging', 'measuring', 'mopping', 'mowing', 'officiating', 'opening', 'operating', 'overflowing', 'packaging', 'packing', 'painting', 'parading', 'paying', 'pedaling', 'peeling', 'performing', 'photographing', 'picking', 'piloting', 'pitching', 'placing', 'planting', 'playing', 'playing fun', 'playing music', 'playing sports', 'playing videogames', 'plugging', 'plunging', 'pointing', 'poking', 'pouring', 'praying', 'preaching', 'pressing', 'protesting', 'pulling', 'punching', 'punting', 'pushing', 'putting', 'queuing', 'racing', 'rafting', 'raining', 'raising', 'reaching', 'reading', 'removing', 'repairing', 'resting', 'riding', 'rinsing', 'rising', 'roaring', 'rocking', 'rolling', 'rowing', 'rubbing', 'running', 'sailing', 'saluting', 'sanding', 'sawing', 'scratching', 'screwing', 'scrubbing', 'selling', 'serving', 'sewing', 'shaking', 'shaving', 'shooting', 'shopping', 'shouting', 'shoveling', 'shredding', 'shrugging', 'signing', 'singing', 'sitting', 'skating', 'sketching', 'skiing', 'skipping', 'slapping', 'sleeping', 'slicing', 'sliding', 'slipping', 'smashing', 'smelling', 'smiling', 'smoking', 'snapping', 'sneezing', 'sniffing', 'snowing', 'snuggling', 'socializing', 'sowing', 'speaking', 'spilling', 'spinning', 'spitting', 'splashing', 'spraying', 'spreading', 'sprinkling', 'sprinting', 'squatting', 'squinting', 'stacking', 'standing', 'starting', 'stealing', 'steering', 'stirring', 'stitching', 'stomping', 'stopping', 'storming', 'stretching', 'stroking', 'studying', 'submerging', 'surfing', 'sweeping', 'swerving', 'swimming', 'swinging', 'talking', 'taping', 'tapping', 'tattooing', 'teaching', 'tearing', 'telephoning', 'throwing', 'tickling', 'towing', 'trimming', 'tripping', 'tuning', 'turning', 'twisting', 'tying', 'typing', 'unloading', 'unpacking', 'vacuuming', 'waking', 'walking', 'washing', 'watering', 'waving', 'waxing', 'weeding', 'welding', 'wetting', 'whistling', 'winking', 'working', 'wrapping', 'wrestling', 'writing', 'yawning']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = ClipTokenizer()\n",
    "class_names_tokenized = [torch.tensor(tokenizer.encode(x + \" <|endoftext|>\")) for x in class_names]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "shared_target_set = {\n",
    "    'data': class_names_tokenized,\n",
    "    'modality': 'text'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the target set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This file location should be specified in the \"SHARED_TARGETS\" section of the config file.\n",
    "with open('/nfs/zhujinguo/datasets/open_source_dataset/MiT_class_name_CLIP_with_endoftext.pkl', 'wb') as f:\n",
    "    pickle.dump(shared_target_set, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7.GLUE datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the order is crucial.\n",
    "class_names_dict =  {\n",
    "    'SST-2': [\"bad\",  \"wonderful\"],\n",
    "    \"CoLA\":  [ \"incorrect\", \"correct\" ],\n",
    "    \"RTE\":   [ \"no\", \"yes\",],\n",
    "    \"MRPC\":  [ \"no\", \"yes\",],\n",
    "    \"QQP\":   [ \"no\", \"yes\",],\n",
    "    \"QNLI\":  [ \"no\", \"yes\",],\n",
    "    \"MNLI\":  [\"no\", \"maybe\", \"yes\"]\n",
    "}\n",
    "tokenizer = ClipTokenizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset_name, class_names in class_names_dict.items():\n",
    "    print(f'{dataset_name}, {class_names}')\n",
    "    class_names_tokenized = [torch.tensor(tokenizer.encode(x + \" <|endoftext|>\")) for x in class_names]\n",
    "    # Add the metadata\n",
    "    shared_target_set = {\n",
    "    'data': class_names_tokenized,\n",
    "    'modality': 'text'\n",
    "    }\n",
    "    # This file location should be specified in the \"SHARED_TARGETS\" section of the config file.\n",
    "    with open(f'/nfs/zhujinguo/datasets/open_source_dataset/bert_pretrain_data/glue_data/{dataset_name}_class_name_CLIP_with_endoftext.pkl', 'wb') as f:\n",
    "        pickle.dump(shared_target_set, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8.VQA All Answers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " 'woods',\n",
       " 'trash can',\n",
       " 'hanging',\n",
       " 'wooden',\n",
       " 'cooking',\n",
       " 'china',\n",
       " 'kids',\n",
       " 'bike rack',\n",
       " 'on phone',\n",
       " 'music',\n",
       " 'travel',\n",
       " 'tulip',\n",
       " 'arrow',\n",
       " 'branch',\n",
       " 'chevron',\n",
       " 'mouth',\n",
       " 'on right',\n",
       " 'rice',\n",
       " 'plate',\n",
       " 'lots',\n",
       " 'nature',\n",
       " 'fruits',\n",
       " 'throwing frisbee',\n",
       " 'blonde',\n",
       " 'life jacket',\n",
       " 'ham',\n",
       " 'hay',\n",
       " 'hat',\n",
       " 'to get to other side',\n",
       " '12:35',\n",
       " 'shadow',\n",
       " '12:30',\n",
       " 'crown',\n",
       " 'crows',\n",
       " 'bottom',\n",
       " 'fish',\n",
       " 'benches',\n",
       " 'fabric',\n",
       " 'congratulations',\n",
       " 'passenger',\n",
       " 'ana',\n",
       " 'triangles',\n",
       " 'roll',\n",
       " 'chain',\n",
       " 'chair',\n",
       " 'crates',\n",
       " '11:45',\n",
       " 'shirts',\n",
       " 'mirrors',\n",
       " 'parachute',\n",
       " 'surfing',\n",
       " 'mango',\n",
       " 'years',\n",
       " 'apron',\n",
       " 'entering',\n",
       " '6:20',\n",
       " '6:25',\n",
       " 'in her hand',\n",
       " 'night time',\n",
       " 'zoo',\n",
       " 'printer',\n",
       " 'suitcases',\n",
       " 's',\n",
       " '4:20',\n",
       " 'wine tasting',\n",
       " 'west',\n",
       " 'newspaper',\n",
       " 'wires',\n",
       " 'singapore',\n",
       " 'advertisement',\n",
       " 'for photo',\n",
       " 'multi colored',\n",
       " 'steamed',\n",
       " 'traffic',\n",
       " 'brushing',\n",
       " 'stagecoach',\n",
       " 'thailand',\n",
       " 'pelicans',\n",
       " 'pull',\n",
       " 'dirty',\n",
       " 'rust',\n",
       " 'tennis shoes',\n",
       " 'cream',\n",
       " 'puppy',\n",
       " 'in box',\n",
       " 'waving',\n",
       " 'ceramic',\n",
       " 'skate park',\n",
       " 'sand',\n",
       " 'bathing suit',\n",
       " 'on tower',\n",
       " '193',\n",
       " 'fast food',\n",
       " 'clock',\n",
       " 'full',\n",
       " 'adidas',\n",
       " 'more',\n",
       " '100 feet',\n",
       " 'door',\n",
       " 'seagull',\n",
       " 'huge',\n",
       " 'city bus',\n",
       " 'paper',\n",
       " 'signs',\n",
       " 'smiling',\n",
       " 'sauce',\n",
       " 'weeds',\n",
       " 'very tall',\n",
       " 'right side',\n",
       " 'theater',\n",
       " 'emirates',\n",
       " 'graffiti',\n",
       " 'yellow and red',\n",
       " 'back left',\n",
       " 'visilab',\n",
       " 'singles',\n",
       " 'barrel',\n",
       " 'blender',\n",
       " 'sunny',\n",
       " 'caramel',\n",
       " 'analog',\n",
       " 'snowboarder',\n",
       " 'bell',\n",
       " 'belt',\n",
       " 'awake',\n",
       " '37',\n",
       " 'cheddar',\n",
       " 'emergency',\n",
       " 'couple',\n",
       " 'harry potter',\n",
       " 'china airlines',\n",
       " 'hawaiian',\n",
       " 'lufthansa',\n",
       " 'land',\n",
       " 'windowsill',\n",
       " 'video',\n",
       " 'scania',\n",
       " 'listening to music',\n",
       " 'pencil',\n",
       " 'baby',\n",
       " 'pocket',\n",
       " 'robot',\n",
       " 'spatula',\n",
       " 'mutt',\n",
       " 'snake',\n",
       " '1:55',\n",
       " \"valentine's day\",\n",
       " '1:50',\n",
       " 'glasses',\n",
       " 'books',\n",
       " 'in cabbage town',\n",
       " 'festival',\n",
       " 'big ben',\n",
       " 'stomach',\n",
       " 'school bus',\n",
       " 'mountainous',\n",
       " 'dishes',\n",
       " 'hot dog',\n",
       " 'thanksgiving',\n",
       " 'teddy bear',\n",
       " 'watching tv',\n",
       " 'jar',\n",
       " 'police officer',\n",
       " 'jal',\n",
       " 'tape',\n",
       " 'riding',\n",
       " 'styrofoam',\n",
       " 'frame',\n",
       " 'roman numerals',\n",
       " 'skateboarding',\n",
       " 'staring',\n",
       " 'doorway',\n",
       " 'commuter',\n",
       " 'tattoo',\n",
       " 'off',\n",
       " 'clocks',\n",
       " 'wet',\n",
       " 'street sign',\n",
       " 'pier',\n",
       " 'flying kites',\n",
       " 'pomeranian',\n",
       " 'taking selfie',\n",
       " 'swimming',\n",
       " 'letters',\n",
       " 'cat and dog',\n",
       " 'ponytail',\n",
       " \"don't know\",\n",
       " 'by window',\n",
       " 'water ski',\n",
       " 'panda',\n",
       " 'holding umbrella',\n",
       " 'ski slope',\n",
       " 'plow',\n",
       " 'sweater',\n",
       " 'coins',\n",
       " 'comcast',\n",
       " '1990',\n",
       " 'nuts',\n",
       " 'ladder',\n",
       " 'cell phone',\n",
       " 'arch',\n",
       " '1 foot',\n",
       " 'coffee',\n",
       " '1 in back',\n",
       " 'white and blue',\n",
       " 'safe',\n",
       " 'l',\n",
       " 'turn right',\n",
       " 'sled',\n",
       " 'barrier',\n",
       " 'tree branch',\n",
       " 'on chair',\n",
       " 'john',\n",
       " 'dogs',\n",
       " 'on grass',\n",
       " 'cereal',\n",
       " 'crosstown',\n",
       " 'giraffes',\n",
       " 'gravy',\n",
       " 'germany',\n",
       " 'rectangle',\n",
       " 't shirt',\n",
       " 'away',\n",
       " 'playing game',\n",
       " 'skull and crossbones',\n",
       " 'kitchen',\n",
       " '3 feet',\n",
       " 'onion rings',\n",
       " 'cylinder',\n",
       " 'tissue',\n",
       " 'cone',\n",
       " 'wheel',\n",
       " 'hand',\n",
       " 'yamaha',\n",
       " 'cooler',\n",
       " 'night',\n",
       " 'cirrus',\n",
       " 'bottom right',\n",
       " 'sailboats',\n",
       " 'silverware',\n",
       " 'gun',\n",
       " 'horseback riding',\n",
       " 'slow down',\n",
       " '2:30',\n",
       " '2:35',\n",
       " 'teacher',\n",
       " '3:25',\n",
       " '3:20',\n",
       " 'pillow',\n",
       " 'green and red',\n",
       " 'uphill',\n",
       " 'flat screen',\n",
       " 'produce',\n",
       " 'vases',\n",
       " 'corona',\n",
       " 'serving',\n",
       " 'still',\n",
       " 'tiara',\n",
       " 'top left',\n",
       " 'slacks',\n",
       " 'not',\n",
       " 'now',\n",
       " '5 star',\n",
       " 'do not enter',\n",
       " 'stir fry',\n",
       " 'ring',\n",
       " 'button up',\n",
       " 'coffee pot',\n",
       " 'hsbc',\n",
       " 'apples',\n",
       " 'windsor',\n",
       " 'america',\n",
       " '7:35',\n",
       " 'cauliflower',\n",
       " 'living room',\n",
       " 'jeans',\n",
       " 'tropical',\n",
       " 'skiers',\n",
       " 'playing baseball',\n",
       " 'foam',\n",
       " 'westin',\n",
       " 'taxi',\n",
       " 'neon',\n",
       " 'foot',\n",
       " 'swim',\n",
       " 'blackberry',\n",
       " 'apartments',\n",
       " 'kite flying',\n",
       " 'stars',\n",
       " 'pitcher',\n",
       " 'omelet',\n",
       " '600',\n",
       " 'cabinets',\n",
       " 'not at all',\n",
       " 'poor',\n",
       " 'poop',\n",
       " 'pool',\n",
       " 'red bull',\n",
       " 'tracks',\n",
       " 'all way',\n",
       " 'grill',\n",
       " 'us airways',\n",
       " 'horizontal',\n",
       " 'chopsticks',\n",
       " 'computers',\n",
       " 'evening',\n",
       " 'talking on phone',\n",
       " 'starbucks',\n",
       " 'heavy',\n",
       " 'safety',\n",
       " '7',\n",
       " 'houses',\n",
       " 'dishwasher',\n",
       " 'american',\n",
       " 'horse',\n",
       " 'station',\n",
       " 'on track',\n",
       " 'boredom',\n",
       " 'toward',\n",
       " 'silver and red',\n",
       " 'in grass',\n",
       " 'gray and black',\n",
       " 'tongs',\n",
       " 'close',\n",
       " 'pictures',\n",
       " 'zipper',\n",
       " 'gmc',\n",
       " 'empty',\n",
       " 'juice',\n",
       " '000',\n",
       " 'cherries',\n",
       " 'motorola',\n",
       " 'vests',\n",
       " 'cones',\n",
       " 'no man',\n",
       " 'towards',\n",
       " 'blue and white',\n",
       " 'squirrel',\n",
       " 'at&t',\n",
       " 'working',\n",
       " 'on car',\n",
       " 'e',\n",
       " 'pizza hut',\n",
       " 'wave',\n",
       " 'snowboards',\n",
       " 'jump',\n",
       " 'pancake',\n",
       " 'on sidewalk',\n",
       " 'camera',\n",
       " 'visibility',\n",
       " \"i don't know\",\n",
       " 'on woman',\n",
       " 'toaster oven',\n",
       " 'bandana',\n",
       " 'farm',\n",
       " '10:35',\n",
       " '10:45',\n",
       " 'taking pictures',\n",
       " 'costume',\n",
       " 'slide',\n",
       " 'ankle',\n",
       " '9:50',\n",
       " 'ottoman',\n",
       " 'baggage claim',\n",
       " 'trucks',\n",
       " 'tent',\n",
       " 'kicking',\n",
       " 'looking at phone',\n",
       " 'controller',\n",
       " 'smoking',\n",
       " 'toothbrushes',\n",
       " 'speaker',\n",
       " 'party',\n",
       " '42',\n",
       " 'balloons',\n",
       " 'riding elephant',\n",
       " 'propeller',\n",
       " 'intersection',\n",
       " 'library',\n",
       " 'home',\n",
       " 'grinding',\n",
       " 'blue and gray',\n",
       " 'north',\n",
       " 'strawberries',\n",
       " 'display',\n",
       " 'finch',\n",
       " 'star',\n",
       " 'foil',\n",
       " 'samsung',\n",
       " 'helmets',\n",
       " 'wakeboard',\n",
       " 'men',\n",
       " 'joshua',\n",
       " 'sliced',\n",
       " 'jackets',\n",
       " 'under',\n",
       " 'for fun',\n",
       " 'room',\n",
       " 'roof',\n",
       " 'checkerboard',\n",
       " 'before',\n",
       " 'glazed',\n",
       " 'ascending',\n",
       " 'hoodie',\n",
       " 'blinders',\n",
       " 'in street',\n",
       " 'lettuce',\n",
       " 'frosted',\n",
       " 'ginger',\n",
       " 'san diego',\n",
       " 'blue',\n",
       " 'mario',\n",
       " 'no cat',\n",
       " 'hotel room',\n",
       " 'celery',\n",
       " 'watermelon',\n",
       " 'ears',\n",
       " 'lotion',\n",
       " 'parsley',\n",
       " 'mozzarella',\n",
       " 'basil',\n",
       " 'hot sauce',\n",
       " 'cane',\n",
       " 'radiator',\n",
       " 'yes',\n",
       " 'strap',\n",
       " 'kitesurfing',\n",
       " 'dead',\n",
       " '10 years',\n",
       " 'ham and cheese',\n",
       " 'skateboarder',\n",
       " 'magazine',\n",
       " 'afternoon',\n",
       " 'selfie',\n",
       " 'down',\n",
       " 'tennis',\n",
       " 'batman',\n",
       " 'landing',\n",
       " 'muffins',\n",
       " 'handicap',\n",
       " 'jacket',\n",
       " 'riding bikes',\n",
       " 'father',\n",
       " '0',\n",
       " 'round',\n",
       " 'frosting',\n",
       " 'box',\n",
       " 'boy',\n",
       " 'bow',\n",
       " 'bob',\n",
       " 'sun hat',\n",
       " 'pizza box',\n",
       " 'man in middle',\n",
       " 'tablecloth',\n",
       " 'basket',\n",
       " 'shoes',\n",
       " 'police',\n",
       " 'monitor',\n",
       " 'lunch',\n",
       " 'man made',\n",
       " 'elephants',\n",
       " 'first',\n",
       " '4 ft',\n",
       " 'pastry',\n",
       " 'jungle',\n",
       " '200',\n",
       " 'sheets',\n",
       " 'not high',\n",
       " 'morning',\n",
       " 'seat',\n",
       " 'boundaries',\n",
       " '2:00',\n",
       " 'tour',\n",
       " 'pacifier',\n",
       " 'squash',\n",
       " 'laptops',\n",
       " 'riding motorcycle',\n",
       " 'sleeping',\n",
       " 'westjet',\n",
       " 'outdoor',\n",
       " 'bow tie',\n",
       " 'wii remotes',\n",
       " 'genetics',\n",
       " 'knife',\n",
       " 'pockets',\n",
       " 'harness',\n",
       " 'traffic lights',\n",
       " 'on water',\n",
       " 'on road',\n",
       " 'sitting',\n",
       " 'washing',\n",
       " 'africa',\n",
       " 'dachshund',\n",
       " 'brushing her teeth',\n",
       " 'numbers',\n",
       " 'comfort',\n",
       " 'coming',\n",
       " 'toiletries',\n",
       " 'dragon',\n",
       " 'faucet',\n",
       " 'top',\n",
       " 'fork and spoon',\n",
       " 'urban',\n",
       " '1:05',\n",
       " '1:00',\n",
       " 'couch',\n",
       " 'eggs',\n",
       " '11:10',\n",
       " '11:15',\n",
       " 'opaque',\n",
       " 'motorcycles',\n",
       " 'doubles',\n",
       " 'mouthwash',\n",
       " 'mailbox',\n",
       " 'restaurant',\n",
       " 'baseball bat',\n",
       " 'very big',\n",
       " 'peppers',\n",
       " 'kodak',\n",
       " 'windmill',\n",
       " 'stripe',\n",
       " 'shape',\n",
       " 'cut',\n",
       " 'cup',\n",
       " 'easter',\n",
       " 'candle',\n",
       " '5:18',\n",
       " '5:15',\n",
       " '5:10',\n",
       " 'tennis court',\n",
       " 'planes',\n",
       " 'curb',\n",
       " 'cafe',\n",
       " 'mayonnaise',\n",
       " 'cooked',\n",
       " 'pink and blue',\n",
       " 'dessert',\n",
       " 'legos',\n",
       " 'packing',\n",
       " 'orange and yellow',\n",
       " 'ketchup and mustard',\n",
       " 'mustache',\n",
       " 'horns',\n",
       " 'potato salad',\n",
       " 'backwards',\n",
       " 'yard',\n",
       " 'skateboard',\n",
       " 'yarn',\n",
       " 'bulldog',\n",
       " 'nobody',\n",
       " 'beets',\n",
       " 'mercedes',\n",
       " 'daffodils',\n",
       " 'racket',\n",
       " 'to dry',\n",
       " 'hammock',\n",
       " 'statues',\n",
       " 'denim',\n",
       " 'flashlight',\n",
       " 'carnations',\n",
       " 'hair',\n",
       " 'shaking hands',\n",
       " 'socks',\n",
       " 'female',\n",
       " 'looking at camera',\n",
       " 'coke',\n",
       " 'flip',\n",
       " 'circus',\n",
       " 'dresser',\n",
       " 'cocker spaniel',\n",
       " 'school',\n",
       " 'feathers',\n",
       " 'umbrella',\n",
       " 'luggage',\n",
       " 'leaves',\n",
       " 'to left',\n",
       " '2:05',\n",
       " '400',\n",
       " '3:15',\n",
       " '3:10',\n",
       " 'detroit',\n",
       " 'public market center',\n",
       " 'small',\n",
       " 'soda',\n",
       " 'fire truck',\n",
       " '700',\n",
       " 'schnauzer',\n",
       " 'sparrow',\n",
       " 'fork and knife',\n",
       " 'wedding',\n",
       " 'beads',\n",
       " 'mud',\n",
       " 'mug',\n",
       " 'finger',\n",
       " 'herding',\n",
       " 'news',\n",
       " 'behind fence',\n",
       " 'elm',\n",
       " 't shirt and jeans',\n",
       " 'conference',\n",
       " 'monkey',\n",
       " 'n',\n",
       " 'nike',\n",
       " 'ocean',\n",
       " 'cherry',\n",
       " 'teddy bears',\n",
       " 'wii controllers',\n",
       " 'descending',\n",
       " 'coffee maker',\n",
       " 'khaki',\n",
       " 'dinner',\n",
       " 'toothpicks',\n",
       " 'fern',\n",
       " 'bats',\n",
       " 'sunlight',\n",
       " 'towing',\n",
       " 'kiting',\n",
       " 'setting',\n",
       " 'papers',\n",
       " 'picture',\n",
       " 'football',\n",
       " 'long time',\n",
       " 'posts',\n",
       " 'cloudy',\n",
       " 'pork',\n",
       " 'tuxedo',\n",
       " '1 in middle',\n",
       " 'pickle',\n",
       " 'nursing',\n",
       " 'black and brown',\n",
       " 'sailboat',\n",
       " '2 years',\n",
       " 'platform',\n",
       " 'farmer',\n",
       " 'cutting hair',\n",
       " 'catching',\n",
       " '100 year party ct',\n",
       " 'turn',\n",
       " \"men's\",\n",
       " 'batting',\n",
       " 'surf',\n",
       " 'equestrian',\n",
       " 'wii remote',\n",
       " '1 hour',\n",
       " 'guitar',\n",
       " 'turkey',\n",
       " 'direction',\n",
       " 'pilot',\n",
       " 'case',\n",
       " 'statue',\n",
       " 'peanut butter',\n",
       " 'towel',\n",
       " 'tower',\n",
       " 'competition',\n",
       " 'moon',\n",
       " 'burton',\n",
       " 'on floor',\n",
       " 'barber shop',\n",
       " 'flats',\n",
       " 'grass',\n",
       " 'roses',\n",
       " 'pillows',\n",
       " 'qantas',\n",
       " 'apartment',\n",
       " 'dairy',\n",
       " 'crest',\n",
       " 'sub',\n",
       " 'sun',\n",
       " 'suv',\n",
       " 'christian',\n",
       " 'donuts',\n",
       " 'horses',\n",
       " 'flat',\n",
       " 'flag',\n",
       " 'lighting',\n",
       " 'short',\n",
       " 'shore',\n",
       " '10:50',\n",
       " '10:55',\n",
       " 'soccer',\n",
       " 'clouds',\n",
       " 'alcohol',\n",
       " 'hill',\n",
       " 'snowboarding',\n",
       " 'urinal',\n",
       " 'stork',\n",
       " 'storm',\n",
       " 'store',\n",
       " 'surfboard',\n",
       " 'king',\n",
       " '1 on right',\n",
       " '8:35',\n",
       " 'skyscrapers',\n",
       " 'wilson',\n",
       " 'electric',\n",
       " 'patterned',\n",
       " 'national express',\n",
       " 'opponent',\n",
       " 'triangle',\n",
       " '9:05',\n",
       " 'sweet',\n",
       " 'shirt',\n",
       " '9',\n",
       " 'cement',\n",
       " 'prince',\n",
       " 'ping pong',\n",
       " 'phone',\n",
       " 'playing video game',\n",
       " 'sports',\n",
       " 'sepia',\n",
       " 'rainbow',\n",
       " 'peach',\n",
       " 'peace',\n",
       " 'parking meter',\n",
       " 'windy',\n",
       " 'very high',\n",
       " 'gray and white',\n",
       " 'compaq',\n",
       " 'kangaroo',\n",
       " 'leaving',\n",
       " 'cell phones',\n",
       " 'duck',\n",
       " 'frog',\n",
       " 'parrots',\n",
       " 'hammer time',\n",
       " 'wrist',\n",
       " 'stuffed animal',\n",
       " 'on bed',\n",
       " 'out',\n",
       " 'brown and white',\n",
       " 'organic',\n",
       " 'g',\n",
       " 'tennis racket',\n",
       " 'umbrellas',\n",
       " 'unknown',\n",
       " 'blue and black',\n",
       " 'dreadlocks',\n",
       " 'clip',\n",
       " 'pipe',\n",
       " 'cubs',\n",
       " 'married',\n",
       " 'identification',\n",
       " 'on his face',\n",
       " 'stucco',\n",
       " 'unclear',\n",
       " 'motorbike',\n",
       " 'rooster',\n",
       " 'camel',\n",
       " 'name',\n",
       " \"can't tell\",\n",
       " 'in middle',\n",
       " 'pasta',\n",
       " 'spiral',\n",
       " 'coffee cup',\n",
       " 'cake',\n",
       " 'maroon',\n",
       " 'space shuttle',\n",
       " 'left and right',\n",
       " 'yellow',\n",
       " 'electricity',\n",
       " 'deli',\n",
       " 'dell',\n",
       " 'eagle',\n",
       " 'behind woman',\n",
       " 'military',\n",
       " 'on dresser',\n",
       " 'machine',\n",
       " 'gaming',\n",
       " 'in sky',\n",
       " 'wing',\n",
       " 'wind',\n",
       " 'wine',\n",
       " 'baseball uniform',\n",
       " 'new orleans',\n",
       " 'silver',\n",
       " 'bowls',\n",
       " 'lego',\n",
       " 'guitar hero',\n",
       " 'legs',\n",
       " 'man on left',\n",
       " 'person',\n",
       " 'in stands',\n",
       " 'zebras',\n",
       " 'victoria',\n",
       " 'little girl',\n",
       " 'recliner',\n",
       " 'menu',\n",
       " 'fair',\n",
       " 'looking out window',\n",
       " 'pirate',\n",
       " 'vaio',\n",
       " 'chrome',\n",
       " 'life',\n",
       " 'lift',\n",
       " 'child',\n",
       " 'chili',\n",
       " 'above toilet',\n",
       " 'gray and red',\n",
       " 'babies',\n",
       " 'bird feeder',\n",
       " 'roast beef',\n",
       " 'main street',\n",
       " 'train car',\n",
       " 'pavement',\n",
       " 'steps',\n",
       " 'people',\n",
       " 'fox',\n",
       " 'fog',\n",
       " 'happy',\n",
       " 'tigers',\n",
       " 'lays',\n",
       " '4 way',\n",
       " 'peacock',\n",
       " 'venice',\n",
       " 'ollie',\n",
       " 'subway',\n",
       " 'teal',\n",
       " 'team',\n",
       " 'current',\n",
       " 'no smoking',\n",
       " 'love',\n",
       " 'sunbathing',\n",
       " 'heineken',\n",
       " 'winter',\n",
       " 'elephant',\n",
       " 'wheelchair',\n",
       " '8 feet',\n",
       " 'colorado',\n",
       " 'bagels',\n",
       " 'polka dot',\n",
       " '2 men',\n",
       " 'gatorade',\n",
       " 'arriving',\n",
       " 'bowling',\n",
       " 'not there',\n",
       " 'container',\n",
       " 'dodgers',\n",
       " '12:10',\n",
       " '12:15',\n",
       " 'ostrich',\n",
       " '2',\n",
       " 'typing',\n",
       " 'give way',\n",
       " 'hockey',\n",
       " 'robe',\n",
       " 'bridge',\n",
       " 'barbed wire',\n",
       " 'cantaloupe',\n",
       " 'jets',\n",
       " '1:35',\n",
       " '1:30',\n",
       " 'twin',\n",
       " 'teddy',\n",
       " 'toothpaste',\n",
       " 'vacation',\n",
       " 'orange juice',\n",
       " 'no dog',\n",
       " 'skier',\n",
       " 'orange and black',\n",
       " 'clay',\n",
       " 'tying tie',\n",
       " 'tube',\n",
       " '6:40',\n",
       " '6:45',\n",
       " 'take off',\n",
       " 'bundt',\n",
       " 'aa',\n",
       " 'gone',\n",
       " 'am',\n",
       " '5:45',\n",
       " '5:40',\n",
       " 'for sale',\n",
       " 'dinosaur',\n",
       " '2 hours',\n",
       " 'dead end',\n",
       " 'bottom left',\n",
       " 'using laptop',\n",
       " 'young',\n",
       " 'indoors',\n",
       " 'parking garage',\n",
       " 'pitching',\n",
       " '4:00',\n",
       " '4:05',\n",
       " 'information',\n",
       " 'not very',\n",
       " 'no parking',\n",
       " 'countryside',\n",
       " 'backyard',\n",
       " 'hauling',\n",
       " 'pans',\n",
       " 'rottweiler',\n",
       " 'mexican',\n",
       " 'swimsuit',\n",
       " 'harbor',\n",
       " 'air canada',\n",
       " 'highway',\n",
       " 'porcelain',\n",
       " 'hydrant',\n",
       " 'w',\n",
       " '2 people',\n",
       " 'cartoon',\n",
       " 'obama',\n",
       " 'refrigerators',\n",
       " 'play',\n",
       " 'cover',\n",
       " 'sleeve',\n",
       " 'crosswalk',\n",
       " '1 year',\n",
       " 'whole',\n",
       " 'lanyard',\n",
       " 'hitting',\n",
       " 'fire',\n",
       " 'farmers market',\n",
       " 'white and brown',\n",
       " 'sesame',\n",
       " 'knee pads',\n",
       " 'audi',\n",
       " 'cucumber',\n",
       " 'stainless steel',\n",
       " 'baseball glove',\n",
       " 'santa',\n",
       " 'cobblestone',\n",
       " 'taking picture',\n",
       " 'garden',\n",
       " 'index',\n",
       " 'twins',\n",
       " 'bird',\n",
       " 'leg',\n",
       " 'banana split',\n",
       " 'mariners',\n",
       " 'standing',\n",
       " 'casserole',\n",
       " 'high',\n",
       " 'animal',\n",
       " 'stop',\n",
       " 'wallpaper',\n",
       " 'notebook',\n",
       " 'logitech',\n",
       " 'next to toilet',\n",
       " 'catch frisbee',\n",
       " 'jesus',\n",
       " 'owner',\n",
       " 'steel',\n",
       " '2:55',\n",
       " '2:50',\n",
       " 'soap',\n",
       " 'biker',\n",
       " 'bikes',\n",
       " 'not possible',\n",
       " 'amtrak',\n",
       " 'email',\n",
       " 'black and red',\n",
       " 'cd',\n",
       " 'ripe',\n",
       " 'north face',\n",
       " 'ball',\n",
       " 'dusk',\n",
       " 'bald',\n",
       " 'overalls',\n",
       " 'bananas',\n",
       " '7:55',\n",
       " 'fire hydrant',\n",
       " 'conference room',\n",
       " 'residential',\n",
       " 'to see',\n",
       " 'onion',\n",
       " 'transport',\n",
       " 'fedex',\n",
       " 'seeds',\n",
       " 'bud light',\n",
       " 'delivery',\n",
       " 'construction',\n",
       " 'smooth',\n",
       " 'volvo',\n",
       " 'black and yellow',\n",
       " 'mushroom',\n",
       " 'footprints',\n",
       " 'rural',\n",
       " 'buoy',\n",
       " 'all',\n",
       " 'wreath',\n",
       " 'blanket',\n",
       " \"women's\",\n",
       " 'home plate',\n",
       " 'lizard',\n",
       " 'above',\n",
       " 'smoke',\n",
       " 'indians',\n",
       " 'boats',\n",
       " 'green',\n",
       " 'giraffe',\n",
       " 'on tray',\n",
       " 'lying down',\n",
       " 'in vase',\n",
       " 'sliding',\n",
       " 'pooping',\n",
       " 'train tracks',\n",
       " 'goalie',\n",
       " 'dc',\n",
       " ...]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('/nfs/zhujinguo/datasets/open_source_dataset/VQA/trainval_label2ans.pkl', 'rb') as f:\n",
    "    answers_mapping = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = ClipTokenizer()\n",
    "answer_tokenized = [torch.tensor(tokenizer.encode(x + \" <|endoftext|>\")) for x in answers_mapping]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add meta data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "shared_target_set = {\n",
    "    'data': answer_tokenized,\n",
    "    'modality': 'text'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the target set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This file location should be specified in the \"SHARED_TARGETS\" section of the config file.\n",
    "with open('/nfs/zhujinguo/datasets/open_source_dataset/VQA_Answers_CLIP_with_endoftext.pkl', 'wb') as f:\n",
    "    pickle.dump(shared_target_set, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "vscode": {
   "interpreter": {
    "hash": "a745cf6333d4d8275ecd56c526d26202f2d2beb96e1206fac92576cf98b427be"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
